{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEMO ON MNIST SPLIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from MAS import *\n",
    "from MNIST_split import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
      "Processing...\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "batch_size=100\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} \n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    torchvision.datasets.MNIST('data', train=True, download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=batch_size, shuffle=True, **kwargs)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    torchvision.datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "batch_size=batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "for digits in [[1,2],[3,4],[5,6],[7,8],[9,0]]:\n",
    "   \n",
    "\n",
    "    dsets={}\n",
    "    dsets['train']=    MNIST_Split('data', train=True, download=False,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ]),digits=digits)\n",
    "    dsets['val']=  MNIST_Split('data', train=False, transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ]),digits=digits)\n",
    "    dlabel=''\n",
    "    for i in digits:\n",
    "        dlabel=dlabel+str(i)\n",
    "    torch.save(dsets,'data/Pytorch_MNIST_dataset//split'+dlabel+'_dataset.pth.tar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lr is 0.01\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NET12/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/9\n",
      "----------\n",
      "lr is 0.01\n",
      "LR is set to 0.01\n",
      "train Loss: 0.0008 Acc: 0.9555\n",
      "val Loss: 0.0001 Acc: 0.9940\n",
      "\n",
      "Epoch 1/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0002 Acc: 0.9903\n",
      "val Loss: 0.0001 Acc: 0.9926\n",
      "\n",
      "Epoch 2/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9927\n",
      "val Loss: 0.0001 Acc: 0.9931\n",
      "\n",
      "Epoch 3/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9940\n",
      "val Loss: 0.0001 Acc: 0.9949\n",
      "\n",
      "Epoch 4/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9954\n",
      "val Loss: 0.0001 Acc: 0.9935\n",
      "\n",
      "Epoch 5/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9957\n",
      "val Loss: 0.0001 Acc: 0.9935\n",
      "\n",
      "Epoch 6/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9966\n",
      "val Loss: 0.0001 Acc: 0.9935\n",
      "\n",
      "Epoch 7/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9972\n",
      "val Loss: 0.0001 Acc: 0.9935\n",
      "\n",
      "Epoch 8/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9976\n",
      "val Loss: 0.0001 Acc: 0.9940\n",
      "\n",
      "Epoch 9/9\n",
      "----------\n",
      "lr is 0.01\n",
      "train Loss: 0.0001 Acc: 0.9976\n",
      "val Loss: 0.0001 Acc: 0.9949\n",
      "\n",
      "Training complete in 0m 5s\n",
      "Best val Acc: 0.994924\n"
     ]
    }
   ],
   "source": [
    "#FIRST TASK TRAINING\n",
    "model_path='General_utils/mnist_net.pth.tar'\n",
    "from Finetune_SGD import *\n",
    "digits = [1,2]\n",
    "dlabel=''\n",
    "for i in digits:\n",
    "    dlabel=dlabel+str(i)\n",
    "\n",
    "dataset_path='data/Pytorch_MNIST_dataset//split'+dlabel+'_dataset.pth.tar'\n",
    "\n",
    "exp_dir='exp_dir/SGD_MNIST_NET'+dlabel\n",
    "\n",
    "num_epochs=10\n",
    "\n",
    "\n",
    "fine_tune_SGD(dataset_path=dataset_path, num_epochs=num_epochs,exp_dir=exp_dir,model_path=model_path,lr=0.01,batch_size=200)\n",
    "model_path=os.path.join(exp_dir,'best_model.pth.tar')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initializing param classifier.0.weight\n",
      "initializing param classifier.0.bias\n",
      "initializing param classifier.2.weight\n",
      "initializing param classifier.2.bias\n",
      "initializing param classifier.4.weight\n",
      "initializing param classifier.4.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "batch number  61\n",
      "batch number  62\n",
      "batch number  63\n",
      "classifier.0.weight\n",
      "omega max is 5.685281753540039\n",
      "omega min is 0.000349677779013291\n",
      "omega mean is 0.18067151308059692\n",
      "classifier.0.bias\n",
      "omega max is 2.588287591934204\n",
      "omega min is 0.0010707491310313344\n",
      "omega mean is 0.33683261275291443\n",
      "classifier.2.weight\n",
      "omega max is 5.875715255737305\n",
      "omega min is 0.0\n",
      "omega mean is 0.1620691865682602\n",
      "classifier.2.bias\n",
      "omega max is 2.074061870574951\n",
      "omega min is 0.0\n",
      "omega mean is 0.371077299118042\n",
      "classifier.4.weight\n",
      "omega max is 12.698545455932617\n",
      "omega min is 0.0\n",
      "omega mean is 2.4823594093322754\n",
      "classifier.4.bias\n",
      "omega max is 7.868046760559082\n",
      "omega min is 7.8264336585998535\n",
      "omega mean is 7.847240447998047\n",
      "classifier.0.weight\n",
      "omega max is 5.685281753540039\n",
      "omega min is 0.000349677779013291\n",
      "omega mean is 0.18067151308059692\n",
      "classifier.0.bias\n",
      "omega max is 2.588287591934204\n",
      "omega min is 0.0010707491310313344\n",
      "omega mean is 0.33683261275291443\n",
      "classifier.2.weight\n",
      "omega max is 5.875715255737305\n",
      "omega min is 0.0\n",
      "omega mean is 0.1620691865682602\n",
      "classifier.2.bias\n",
      "omega max is 2.074061870574951\n",
      "omega min is 0.0\n",
      "omega mean is 0.371077299118042\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NET34/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.01\n",
      "train Loss: 0.0008 Acc: 0.9567\n",
      "val Loss: 0.0002 Acc: 0.9950\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9932\n",
      "val Loss: 0.0002 Acc: 0.9955\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9946\n",
      "val Loss: 0.0002 Acc: 0.9970\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9952\n",
      "val Loss: 0.0001 Acc: 0.9980\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9959\n",
      "val Loss: 0.0001 Acc: 0.9970\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9972\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9971\n",
      "val Loss: 0.0001 Acc: 0.9970\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9972\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9971\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9972\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9971\n",
      "val Loss: 0.0001 Acc: 0.9955\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9976\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9980\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9983\n",
      "val Loss: 0.0001 Acc: 0.9980\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9985\n",
      "val Loss: 0.0001 Acc: 0.9980\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9987\n",
      "val Loss: 0.0001 Acc: 0.9980\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9984\n",
      "val Loss: 0.0001 Acc: 0.9980\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9986\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9988\n",
      "val Loss: 0.0001 Acc: 0.9975\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0000 Acc: 0.9991\n",
      "val Loss: 0.0001 Acc: 0.9980\n",
      "\n",
      "Training complete in 0m 11s\n",
      "Best val Acc: 0.997992\n",
      "initializing param classifier.0.weight\n",
      "initializing param classifier.0.bias\n",
      "initializing param classifier.2.weight\n",
      "initializing param classifier.2.bias\n",
      "initializing param classifier.4.weight\n",
      "initializing param classifier.4.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "batch number  61\n",
      "batch number  62\n",
      "batch number  63\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "classifier.0.weight\n",
      "omega max is 7.367978096008301\n",
      "omega min is 0.001319059869274497\n",
      "omega mean is 0.41499391198158264\n",
      "classifier.0.bias\n",
      "omega max is 3.929434061050415\n",
      "omega min is 0.0034162201918661594\n",
      "omega mean is 0.7643951177597046\n",
      "classifier.2.weight\n",
      "omega max is 12.59623908996582\n",
      "omega min is 0.0\n",
      "omega mean is 0.42077699303627014\n",
      "classifier.2.bias\n",
      "omega max is 4.091132640838623\n",
      "omega min is 7.032184657873586e-05\n",
      "omega mean is 0.9292923212051392\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "classifier.0.weight\n",
      "omega max is 7.367978096008301\n",
      "omega min is 0.001319059869274497\n",
      "omega mean is 0.41499391198158264\n",
      "classifier.0.bias\n",
      "omega max is 3.929434061050415\n",
      "omega min is 0.0034162201918661594\n",
      "omega mean is 0.7643951177597046\n",
      "classifier.2.weight\n",
      "omega max is 12.59623908996582\n",
      "omega min is 0.0\n",
      "omega mean is 0.42077699303627014\n",
      "classifier.2.bias\n",
      "omega max is 4.091132640838623\n",
      "omega min is 7.032184657873586e-05\n",
      "omega mean is 0.9292923212051392\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NET56/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train Loss: 0.0010 Acc: 0.9332\n",
      "val Loss: 0.0006 Acc: 0.9692\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9740\n",
      "val Loss: 0.0005 Acc: 0.9773\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9789\n",
      "val Loss: 0.0005 Acc: 0.9757\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9816\n",
      "val Loss: 0.0004 Acc: 0.9784\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9852\n",
      "val Loss: 0.0004 Acc: 0.9816\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9857\n",
      "val Loss: 0.0003 Acc: 0.9816\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9862\n",
      "val Loss: 0.0003 Acc: 0.9849\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9875\n",
      "val Loss: 0.0003 Acc: 0.9832\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9884\n",
      "val Loss: 0.0003 Acc: 0.9811\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9903\n",
      "val Loss: 0.0003 Acc: 0.9843\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9915\n",
      "val Loss: 0.0003 Acc: 0.9827\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9919\n",
      "val Loss: 0.0003 Acc: 0.9838\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9905\n",
      "val Loss: 0.0003 Acc: 0.9854\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9888\n",
      "val Loss: 0.0003 Acc: 0.9832\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9924\n",
      "val Loss: 0.0003 Acc: 0.9822\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9915\n",
      "val Loss: 0.0003 Acc: 0.9876\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9941\n",
      "val Loss: 0.0003 Acc: 0.9843\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9929\n",
      "val Loss: 0.0003 Acc: 0.9870\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9919\n",
      "val Loss: 0.0004 Acc: 0.9816\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9935\n",
      "val Loss: 0.0003 Acc: 0.9865\n",
      "\n",
      "Training complete in 0m 12s\n",
      "Best val Acc: 0.987568\n",
      "initializing param classifier.0.weight\n",
      "initializing param classifier.0.bias\n",
      "initializing param classifier.2.weight\n",
      "initializing param classifier.2.bias\n",
      "initializing param classifier.4.weight\n",
      "initializing param classifier.4.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "batch number  61\n",
      "batch number  62\n",
      "batch number  63\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "classifier.0.weight\n",
      "omega max is 12.482980728149414\n",
      "omega min is 0.009099539369344711\n",
      "omega mean is 0.7587667107582092\n",
      "classifier.0.bias\n",
      "omega max is 7.461484909057617\n",
      "omega min is 0.026722995564341545\n",
      "omega mean is 1.392875075340271\n",
      "classifier.2.weight\n",
      "omega max is 24.051990509033203\n",
      "omega min is 0.0\n",
      "omega mean is 0.828478217124939\n",
      "classifier.2.bias\n",
      "omega max is 8.133673667907715\n",
      "omega min is 0.00010556702181929722\n",
      "omega mean is 1.8054494857788086\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "classifier.0.weight\n",
      "omega max is 12.482980728149414\n",
      "omega min is 0.009099539369344711\n",
      "omega mean is 0.7587667107582092\n",
      "classifier.0.bias\n",
      "omega max is 7.461484909057617\n",
      "omega min is 0.026722995564341545\n",
      "omega mean is 1.392875075340271\n",
      "classifier.2.weight\n",
      "omega max is 24.051990509033203\n",
      "omega min is 0.0\n",
      "omega mean is 0.828478217124939\n",
      "classifier.2.bias\n",
      "omega max is 8.133673667907715\n",
      "omega min is 0.00010556702181929722\n",
      "omega mean is 1.8054494857788086\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NET78/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.01\n",
      "train Loss: 0.0011 Acc: 0.9497\n",
      "val Loss: 0.0006 Acc: 0.9760\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9848\n",
      "val Loss: 0.0004 Acc: 0.9810\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9876\n",
      "val Loss: 0.0004 Acc: 0.9850\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9888\n",
      "val Loss: 0.0003 Acc: 0.9870\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9899\n",
      "val Loss: 0.0003 Acc: 0.9880\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9898\n",
      "val Loss: 0.0003 Acc: 0.9865\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9896\n",
      "val Loss: 0.0003 Acc: 0.9860\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9910\n",
      "val Loss: 0.0002 Acc: 0.9890\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9898\n",
      "val Loss: 0.0003 Acc: 0.9845\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9899\n",
      "val Loss: 0.0003 Acc: 0.9855\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9896\n",
      "val Loss: 0.0002 Acc: 0.9890\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9917\n",
      "val Loss: 0.0002 Acc: 0.9920\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9917\n",
      "val Loss: 0.0002 Acc: 0.9880\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9889\n",
      "val Loss: 0.0002 Acc: 0.9870\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9909\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val Loss: 0.0004 Acc: 0.9805\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9927\n",
      "val Loss: 0.0002 Acc: 0.9905\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9909\n",
      "val Loss: 0.0002 Acc: 0.9910\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9918\n",
      "val Loss: 0.0004 Acc: 0.9815\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9922\n",
      "val Loss: 0.0002 Acc: 0.9885\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9921\n",
      "val Loss: 0.0002 Acc: 0.9875\n",
      "\n",
      "Training complete in 0m 12s\n",
      "Best val Acc: 0.992008\n",
      "initializing param classifier.0.weight\n",
      "initializing param classifier.0.bias\n",
      "initializing param classifier.2.weight\n",
      "initializing param classifier.2.bias\n",
      "initializing param classifier.4.weight\n",
      "initializing param classifier.4.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "batch number  61\n",
      "batch number  62\n",
      "batch number  63\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "classifier.0.weight\n",
      "omega max is 14.715436935424805\n",
      "omega min is 0.007146044168621302\n",
      "omega mean is 1.1208385229110718\n",
      "classifier.0.bias\n",
      "omega max is 8.53923225402832\n",
      "omega min is 0.02047429047524929\n",
      "omega mean is 2.0385072231292725\n",
      "classifier.2.weight\n",
      "omega max is 30.052894592285156\n",
      "omega min is 0.0\n",
      "omega mean is 1.2649389505386353\n",
      "classifier.2.bias\n",
      "omega max is 11.109643936157227\n",
      "omega min is 0.0\n",
      "omega mean is 2.7382564544677734\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "classifier.0.weight\n",
      "omega max is 14.715436935424805\n",
      "omega min is 0.007146044168621302\n",
      "omega mean is 1.1208385229110718\n",
      "classifier.0.bias\n",
      "omega max is 8.53923225402832\n",
      "omega min is 0.02047429047524929\n",
      "omega mean is 2.0385072231292725\n",
      "classifier.2.weight\n",
      "omega max is 30.052894592285156\n",
      "omega min is 0.0\n",
      "omega mean is 1.2649389505386353\n",
      "classifier.2.bias\n",
      "omega max is 11.109643936157227\n",
      "omega min is 0.0\n",
      "omega mean is 2.7382564544677734\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NET90/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.01\n",
      "train Loss: 0.0007 Acc: 0.9661\n",
      "val Loss: 0.0004 Acc: 0.9804\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9849\n",
      "val Loss: 0.0003 Acc: 0.9849\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9865\n",
      "val Loss: 0.0003 Acc: 0.9884\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9884\n",
      "val Loss: 0.0003 Acc: 0.9899\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9886\n",
      "val Loss: 0.0003 Acc: 0.9884\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9888\n",
      "val Loss: 0.0002 Acc: 0.9915\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9900\n",
      "val Loss: 0.0002 Acc: 0.9925\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9896\n",
      "val Loss: 0.0002 Acc: 0.9930\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9913\n",
      "val Loss: 0.0002 Acc: 0.9920\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9914\n",
      "val Loss: 0.0002 Acc: 0.9920\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9940\n",
      "val Loss: 0.0001 Acc: 0.9950\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9944\n",
      "val Loss: 0.0002 Acc: 0.9930\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train Loss: 0.0001 Acc: 0.9945\n",
      "val Loss: 0.0001 Acc: 0.9945\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9949\n",
      "val Loss: 0.0001 Acc: 0.9930\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9941\n",
      "val Loss: 0.0001 Acc: 0.9945\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9957\n",
      "val Loss: 0.0001 Acc: 0.9950\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9949\n",
      "val Loss: 0.0002 Acc: 0.9935\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9954\n",
      "val Loss: 0.0001 Acc: 0.9950\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9954\n",
      "val Loss: 0.0001 Acc: 0.9940\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9956\n",
      "val Loss: 0.0001 Acc: 0.9935\n",
      "\n",
      "Training complete in 0m 12s\n",
      "Best val Acc: 0.994972\n"
     ]
    }
   ],
   "source": [
    "#MIMIC the case when samples from the previous takss are seen in each step\n",
    "from MAS import *\n",
    "all_digits=[[3,4],[5,6],[7,8],[9,0]]\n",
    "reg_sets=[]\n",
    "dataset_path='data/Pytorch_MNIST_dataset//split12_dataset.pth.tar'\n",
    "\n",
    "exp_dir='exp_dir/SGD_MNIST_NET12'\n",
    "pevious_pathes=[]\n",
    "reg_lambda=1\n",
    "for digits in all_digits:\n",
    "    reg_sets.append(dataset_path)\n",
    "    model_path=os.path.join(exp_dir,'best_model.pth.tar')\n",
    "    pevious_pathes.append(model_path)\n",
    "    dlabel=''\n",
    "    for i in digits:\n",
    "        dlabel=dlabel+str(i)\n",
    "\n",
    "    dataset_path='data/Pytorch_MNIST_dataset//split'+dlabel+'_dataset.pth.tar'\n",
    "\n",
    "    exp_dir='exp_dir/SGD_MNIST_NET'+dlabel\n",
    "   \n",
    "    num_epochs=20\n",
    "    data_dirs=None\n",
    "\n",
    "\n",
    "\n",
    "    MAS_sequence(dataset_path=dataset_path,pevious_pathes=pevious_pathes,previous_task_model_path=model_path,exp_dir=exp_dir,data_dirs=data_dirs,reg_sets=reg_sets,reg_lambda=reg_lambda,batch_size=200,num_epochs=num_epochs,lr=1e-2,norm='L2',b1=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 99.4000922935\n",
      "Accuracy: 99.4923857868\n",
      "Accuracy: 99.7991967871\n",
      "Accuracy: 99.7991967871\n",
      "Accuracy: 98.4324324324\n",
      "Accuracy: 98.7567567568\n",
      "Accuracy: 99.2007992008\n",
      "Accuracy: 99.2007992008\n",
      "Accuracy: 99.4972347914\n",
      "Accuracy: 99.4972347914\n",
      "0.0833235635266\n"
     ]
    }
   ],
   "source": [
    "from MAS import *\n",
    "#estimate forgetting\n",
    "all_digits=[[1,2],[3,4],[5,6],[7,8],[9,0]]\n",
    "average_forgetting=0\n",
    "exp_dir='exp_dir/SGD_MNIST_NET12'\n",
    "from Test_sequential  import *\n",
    "for digits in all_digits:\n",
    "    dlabel=''\n",
    "    for i in digits:\n",
    "        dlabel=dlabel+str(i)\n",
    "    if all_digits.index(digits)>0:\n",
    "        exp_dir='exp_dir/SGD_MNIST_NET'+dlabel\n",
    "    previous_model_path=os.path.join(exp_dir,'best_model.pth.tar')\n",
    "    exp_dir='exp_dir/SGD_MNIST_NET90'\n",
    "    dataset_path='data/Pytorch_MNIST_dataset//split'+dlabel+'_dataset.pth.tar'\n",
    "    current_model_path=os.path.join(exp_dir,'best_model.pth.tar')\n",
    "    acc2=test_seq_task_performance(previous_model_path=previous_model_path,current_model_path=current_model_path,dataset_path=dataset_path)\n",
    "    acc1=test_model(previous_model_path,dataset_path)\n",
    "    forgetting=acc1-acc2\n",
    "    average_forgetting=average_forgetting+forgetting\n",
    "average_forgetting=average_forgetting/len(all_digits)    \n",
    "print(average_forgetting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initializing param classifier.0.weight\n",
      "initializing param classifier.0.bias\n",
      "initializing param classifier.2.weight\n",
      "initializing param classifier.2.bias\n",
      "initializing param classifier.4.weight\n",
      "initializing param classifier.4.bias\n",
      "********************MAS with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "batch number  61\n",
      "batch number  62\n",
      "batch number  63\n",
      "classifier.0.weight\n",
      "omega max is 7.547848224639893\n",
      "omega min is 0.0003669970319606364\n",
      "omega mean is 0.26486775279045105\n",
      "classifier.0.bias\n",
      "omega max is 3.3869268894195557\n",
      "omega min is 0.0011362407822161913\n",
      "omega mean is 0.5008917450904846\n",
      "classifier.2.weight\n",
      "omega max is 6.611863136291504\n",
      "omega min is 0.0\n",
      "omega mean is 0.20741108059883118\n",
      "classifier.2.bias\n",
      "omega max is 2.5368030071258545\n",
      "omega min is 0.0\n",
      "omega mean is 0.4908902049064636\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NETACC34/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.001\n",
      "train Loss: 0.0023 Acc: 0.8871\n",
      "val Loss: 0.0009 Acc: 0.9669\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0007 Acc: 0.9708\n",
      "val Loss: 0.0005 Acc: 0.9849\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9811\n",
      "val Loss: 0.0004 Acc: 0.9895\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9846\n",
      "val Loss: 0.0003 Acc: 0.9920\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9867\n",
      "val Loss: 0.0003 Acc: 0.9925\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9887\n",
      "val Loss: 0.0003 Acc: 0.9930\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9906\n",
      "val Loss: 0.0003 Acc: 0.9940\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9910\n",
      "val Loss: 0.0002 Acc: 0.9935\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9921\n",
      "val Loss: 0.0002 Acc: 0.9935\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9923\n",
      "val Loss: 0.0002 Acc: 0.9945\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9926\n",
      "val Loss: 0.0002 Acc: 0.9945\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9927\n",
      "val Loss: 0.0002 Acc: 0.9950\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9928\n",
      "val Loss: 0.0002 Acc: 0.9945\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9927\n",
      "val Loss: 0.0002 Acc: 0.9945\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9934\n",
      "val Loss: 0.0002 Acc: 0.9945\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9941\n",
      "val Loss: 0.0002 Acc: 0.9945\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9943\n",
      "val Loss: 0.0001 Acc: 0.9950\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9947\n",
      "val Loss: 0.0001 Acc: 0.9955\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9948\n",
      "val Loss: 0.0001 Acc: 0.9945\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0001 Acc: 0.9945\n",
      "val Loss: 0.0001 Acc: 0.9945\n",
      "\n",
      "Training complete in 0m 12s\n",
      "Best val Acc: 0.995482\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "classifier.0.weight\n",
      "omega max is 8.4379243850708\n",
      "omega min is 0.0007892365101724863\n",
      "omega mean is 0.42541417479515076\n",
      "classifier.0.bias\n",
      "omega max is 4.050810813903809\n",
      "omega min is 0.0022065634839236736\n",
      "omega mean is 0.79400235414505\n",
      "classifier.2.weight\n",
      "omega max is 9.642128944396973\n",
      "omega min is 0.0\n",
      "omega mean is 0.37501978874206543\n",
      "classifier.2.bias\n",
      "omega max is 4.1957621574401855\n",
      "omega min is 6.4411810853926e-06\n",
      "omega mean is 0.8574461340904236\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NETACC56/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.001\n",
      "train Loss: 0.0022 Acc: 0.8601\n",
      "val Loss: 0.0011 Acc: 0.9486\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0009 Acc: 0.9553\n",
      "val Loss: 0.0008 Acc: 0.9649\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0007 Acc: 0.9645\n",
      "val Loss: 0.0007 Acc: 0.9697\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0006 Acc: 0.9706\n",
      "val Loss: 0.0006 Acc: 0.9719\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0006 Acc: 0.9729\n",
      "val Loss: 0.0006 Acc: 0.9735\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0006 Acc: 0.9743\n",
      "val Loss: 0.0006 Acc: 0.9746\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9754\n",
      "val Loss: 0.0005 Acc: 0.9773\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9765\n",
      "val Loss: 0.0005 Acc: 0.9762\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9782\n",
      "val Loss: 0.0005 Acc: 0.9773\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9787\n",
      "val Loss: 0.0005 Acc: 0.9789\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9801\n",
      "val Loss: 0.0005 Acc: 0.9789\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9798\n",
      "val Loss: 0.0005 Acc: 0.9795\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9802\n",
      "val Loss: 0.0005 Acc: 0.9789\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9810\n",
      "val Loss: 0.0005 Acc: 0.9800\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9806\n",
      "val Loss: 0.0004 Acc: 0.9805\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9810\n",
      "val Loss: 0.0004 Acc: 0.9800\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9822\n",
      "val Loss: 0.0004 Acc: 0.9795\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9823\n",
      "val Loss: 0.0004 Acc: 0.9795\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9820\n",
      "val Loss: 0.0005 Acc: 0.9805\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9835\n",
      "val Loss: 0.0004 Acc: 0.9800\n",
      "\n",
      "Training complete in 0m 11s\n",
      "Best val Acc: 0.980541\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "classifier.0.weight\n",
      "omega max is 9.325267791748047\n",
      "omega min is 0.004173568449914455\n",
      "omega mean is 0.5732074975967407\n",
      "classifier.0.bias\n",
      "omega max is 4.7266845703125\n",
      "omega min is 0.02169070392847061\n",
      "omega mean is 1.0610427856445312\n",
      "classifier.2.weight\n",
      "omega max is 11.732137680053711\n",
      "omega min is 0.0\n",
      "omega mean is 0.5376771092414856\n",
      "classifier.2.bias\n",
      "omega max is 5.129436492919922\n",
      "omega min is 8.648231414554175e-06\n",
      "omega mean is 1.1808068752288818\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NETACC78/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.001\n",
      "train Loss: 0.0024 Acc: 0.8707\n",
      "val Loss: 0.0012 Acc: 0.9550\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0008 Acc: 0.9746\n",
      "val Loss: 0.0008 Acc: 0.9705\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0006 Acc: 0.9809\n",
      "val Loss: 0.0007 Acc: 0.9745\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9833\n",
      "val Loss: 0.0006 Acc: 0.9780\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9846\n",
      "val Loss: 0.0005 Acc: 0.9790\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9859\n",
      "val Loss: 0.0005 Acc: 0.9805\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9867\n",
      "val Loss: 0.0004 Acc: 0.9820\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9874\n",
      "val Loss: 0.0004 Acc: 0.9820\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9881\n",
      "val Loss: 0.0004 Acc: 0.9805\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9879\n",
      "val Loss: 0.0004 Acc: 0.9830\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9885\n",
      "val Loss: 0.0004 Acc: 0.9845\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9892\n",
      "val Loss: 0.0004 Acc: 0.9840\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9894\n",
      "val Loss: 0.0004 Acc: 0.9835\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9901\n",
      "val Loss: 0.0003 Acc: 0.9835\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9903\n",
      "val Loss: 0.0004 Acc: 0.9845\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9903\n",
      "val Loss: 0.0003 Acc: 0.9845\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9912\n",
      "val Loss: 0.0003 Acc: 0.9840\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9912\n",
      "val Loss: 0.0003 Acc: 0.9845\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9908\n",
      "val Loss: 0.0003 Acc: 0.9850\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9915\n",
      "val Loss: 0.0003 Acc: 0.9850\n",
      "\n",
      "Training complete in 0m 12s\n",
      "Best val Acc: 0.985015\n",
      "storing previous omega classifier.0.weight\n",
      "storing previous omega classifier.0.bias\n",
      "storing previous omega classifier.2.weight\n",
      "storing previous omega classifier.2.bias\n",
      "********************objective with L2 norm***************\n",
      "dictoinary length1\n",
      "batch number  0\n",
      "batch number  1\n",
      "batch number  2\n",
      "batch number  3\n",
      "batch number  4\n",
      "batch number  5\n",
      "batch number  6\n",
      "batch number  7\n",
      "batch number  8\n",
      "batch number  9\n",
      "batch number  10\n",
      "batch number  11\n",
      "batch number  12\n",
      "batch number  13\n",
      "batch number  14\n",
      "batch number  15\n",
      "batch number  16\n",
      "batch number  17\n",
      "batch number  18\n",
      "batch number  19\n",
      "batch number  20\n",
      "batch number  21\n",
      "batch number  22\n",
      "batch number  23\n",
      "batch number  24\n",
      "batch number  25\n",
      "batch number  26\n",
      "batch number  27\n",
      "batch number  28\n",
      "batch number  29\n",
      "batch number  30\n",
      "batch number  31\n",
      "batch number  32\n",
      "batch number  33\n",
      "batch number  34\n",
      "batch number  35\n",
      "batch number  36\n",
      "batch number  37\n",
      "batch number  38\n",
      "batch number  39\n",
      "batch number  40\n",
      "batch number  41\n",
      "batch number  42\n",
      "batch number  43\n",
      "batch number  44\n",
      "batch number  45\n",
      "batch number  46\n",
      "batch number  47\n",
      "batch number  48\n",
      "batch number  49\n",
      "batch number  50\n",
      "batch number  51\n",
      "batch number  52\n",
      "batch number  53\n",
      "batch number  54\n",
      "batch number  55\n",
      "batch number  56\n",
      "batch number  57\n",
      "batch number  58\n",
      "batch number  59\n",
      "batch number  60\n",
      "restoring previous omega classifier.0.weight\n",
      "restoring previous omega classifier.0.bias\n",
      "restoring previous omega classifier.2.weight\n",
      "restoring previous omega classifier.2.bias\n",
      "classifier.0.weight\n",
      "omega max is 9.6002197265625\n",
      "omega min is 0.009012761525809765\n",
      "omega mean is 0.7675426006317139\n",
      "classifier.0.bias\n",
      "omega max is 5.095241069793701\n",
      "omega min is 0.026017343625426292\n",
      "omega mean is 1.4048792123794556\n",
      "classifier.2.weight\n",
      "omega max is 12.50429916381836\n",
      "omega min is 0.0\n",
      "omega mean is 0.7640946507453918\n",
      "classifier.2.bias\n",
      "omega max is 5.46642541885376\n",
      "omega min is 0.00011606516636675224\n",
      "omega mean is 1.600907564163208\n",
      "classifier.4.weight\n",
      "classifier.4.bias\n",
      "dictoinary length2\n",
      "=> no checkpoint found at 'exp_dir/SGD_MNIST_NETACC90/epoch.pth.tar'\n",
      "0\n",
      "Epoch 0/19\n",
      "----------\n",
      "LR is set to 0.001\n",
      "train Loss: 0.0015 Acc: 0.9314\n",
      "val Loss: 0.0006 Acc: 0.9799\n",
      "\n",
      "Epoch 1/19\n",
      "----------\n",
      "train Loss: 0.0005 Acc: 0.9829\n",
      "val Loss: 0.0005 Acc: 0.9814\n",
      "\n",
      "Epoch 2/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9843\n",
      "val Loss: 0.0005 Acc: 0.9839\n",
      "\n",
      "Epoch 3/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9858\n",
      "val Loss: 0.0004 Acc: 0.9829\n",
      "\n",
      "Epoch 4/19\n",
      "----------\n",
      "train Loss: 0.0004 Acc: 0.9864\n",
      "val Loss: 0.0004 Acc: 0.9844\n",
      "\n",
      "Epoch 5/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9871\n",
      "val Loss: 0.0004 Acc: 0.9844\n",
      "\n",
      "Epoch 6/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9874\n",
      "val Loss: 0.0004 Acc: 0.9854\n",
      "\n",
      "Epoch 7/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9880\n",
      "val Loss: 0.0004 Acc: 0.9859\n",
      "\n",
      "Epoch 8/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9886\n",
      "val Loss: 0.0004 Acc: 0.9864\n",
      "\n",
      "Epoch 9/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9887\n",
      "val Loss: 0.0004 Acc: 0.9874\n",
      "\n",
      "Epoch 10/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9894\n",
      "val Loss: 0.0004 Acc: 0.9879\n",
      "\n",
      "Epoch 11/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9896\n",
      "val Loss: 0.0003 Acc: 0.9874\n",
      "\n",
      "Epoch 12/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9900\n",
      "val Loss: 0.0003 Acc: 0.9874\n",
      "\n",
      "Epoch 13/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9903\n",
      "val Loss: 0.0003 Acc: 0.9884\n",
      "\n",
      "Epoch 14/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9901\n",
      "val Loss: 0.0003 Acc: 0.9884\n",
      "\n",
      "Epoch 15/19\n",
      "----------\n",
      "train Loss: 0.0003 Acc: 0.9904\n",
      "val Loss: 0.0003 Acc: 0.9894\n",
      "\n",
      "Epoch 16/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9905\n",
      "val Loss: 0.0003 Acc: 0.9899\n",
      "\n",
      "Epoch 17/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9910\n",
      "val Loss: 0.0003 Acc: 0.9889\n",
      "\n",
      "Epoch 18/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9909\n",
      "val Loss: 0.0003 Acc: 0.9904\n",
      "\n",
      "Epoch 19/19\n",
      "----------\n",
      "train Loss: 0.0002 Acc: 0.9908\n",
      "val Loss: 0.0003 Acc: 0.9904\n",
      "\n",
      "Training complete in 0m 12s\n",
      "Best val Acc: 0.990447\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#Mimic the standard setup used when after each task the omega is compute on the training samples and accumelated\n",
    "from MAS import *\n",
    "all_digits=[[3,4],[5,6],[7,8],[9,0]]\n",
    "reg_sets=[]\n",
    "dataset_path='data/Pytorch_MNIST_dataset//split12_dataset.pth.tar'\n",
    "\n",
    "exp_dir='exp_dir/SGD_MNIST_NET12'\n",
    "\n",
    "reg_lambda=1#0.5\n",
    "for digits in all_digits:\n",
    "    reg_sets=[]\n",
    "    reg_sets.append(dataset_path)\n",
    "    model_path=os.path.join(exp_dir,'best_model.pth.tar')\n",
    "    \n",
    "    dlabel=''\n",
    "    for i in digits:\n",
    "        dlabel=dlabel+str(i)\n",
    "\n",
    "    dataset_path='data/Pytorch_MNIST_dataset//split'+dlabel+'_dataset.pth.tar'\n",
    "\n",
    "    exp_dir='exp_dir/SGD_MNIST_NETACC'+dlabel\n",
    "  \n",
    "    num_epochs=20\n",
    "    data_dir=None\n",
    "\n",
    "    if all_digits.index(digits)>0:\n",
    "        MAS_Omega_Acuumelation(dataset_path,previous_task_model_path=model_path,exp_dir=exp_dir,data_dir=data_dir,reg_sets=reg_sets,reg_lambda=reg_lambda,norm='L2', num_epochs=num_epochs,lr=0.1e-2,batch_size=200,b1=False)\n",
    "    else:\n",
    "        MAS(dataset_path,previous_task_model_path=model_path,exp_dir=exp_dir,data_dir=data_dir,reg_sets=reg_sets,reg_lambda=reg_lambda,norm='L2', num_epochs=num_epochs,lr=0.1e-2,batch_size=200,b1=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 99.4923857868\n",
      "Accuracy: 99.4923857868\n",
      "Accuracy: 99.6485943775\n",
      "Accuracy: 99.5481927711\n",
      "Accuracy: 98.0\n",
      "Accuracy: 98.0540540541\n",
      "Accuracy: 98.1018981019\n",
      "Accuracy: 98.5014985015\n",
      "Accuracy: 99.0447461036\n",
      "Accuracy: 99.0447461036\n",
      "0.0706505694457\n"
     ]
    }
   ],
   "source": [
    "#estimate forgetting\n",
    "from MAS import *\n",
    "all_digits=[[1,2],[3,4],[5,6],[7,8],[9,0]]\n",
    "average_forgetting=0\n",
    "exp_dir='exp_dir/SGD_MNIST_NET12'\n",
    "from Test_sequential  import *\n",
    "for digits in all_digits:\n",
    "    dlabel=''\n",
    "    for i in digits:\n",
    "        dlabel=dlabel+str(i)\n",
    "    if all_digits.index(digits)>0:\n",
    "        exp_dir='exp_dir/SGD_MNIST_NETACC'+dlabel\n",
    "    previous_model_path=os.path.join(exp_dir,'best_model.pth.tar')\n",
    "    exp_dir='exp_dir/SGD_MNIST_NETACC90'\n",
    "    dataset_path='data/Pytorch_MNIST_dataset//split'+dlabel+'_dataset.pth.tar'\n",
    "    current_model_path=os.path.join(exp_dir,'best_model.pth.tar')\n",
    "    acc2=test_seq_task_performance(previous_model_path=previous_model_path,current_model_path=current_model_path,dataset_path=dataset_path)\n",
    "    acc1=test_model(previous_model_path,dataset_path)\n",
    "    forgetting=acc1-acc2\n",
    "    average_forgetting=average_forgetting+forgetting\n",
    "average_forgetting=average_forgetting/len(all_digits)    \n",
    "print(average_forgetting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
